package org.rency.crawler.fetch.service.http.impl;

import java.io.IOException;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.net.ssl.SSLHandshakeException;
import javax.security.cert.CertificateException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.RandomUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.NameValuePair;
import org.apache.http.NoHttpResponseException;
import org.apache.http.auth.AUTH;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.MalformedChallengeException;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CookieStore;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.conn.ConnectionPoolTimeoutException;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
import org.apache.http.protocol.HttpCoreContext;
import org.apache.http.util.EntityUtils;
import org.rency.common.utils.domain.Kvp;
import org.rency.crawler.common.CrawlerConstants;
import org.rency.crawler.common.enums.HttpMethod;
import org.rency.crawler.common.exception.CrawlerException;
import org.rency.crawler.common.exception.CrawlerIllegalException;
import org.rency.crawler.domain.http.FetchHttpRequest;
import org.rency.crawler.domain.http.FetchHttpResponse;
import org.rency.crawler.fetch.helper.FetchHelper;
import org.rency.crawler.fetch.repository.MetaResourceRepository;
import org.rency.crawler.fetch.service.http.HttpService;
import org.rency.crawler.plugins.service.UniversalDetectorPlugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

/**
 * 名称:HttpServiceImpl
 * 描述:${TODO}
 *
 * @author:rency
 * @version:V1.0.0
 * @JDK 1.8
 * @since 2017/11/23 21:33
 */
@Service
public class HttpServiceImpl implements HttpService,CrawlerConstants {

  private static final Logger logger = LoggerFactory.getLogger(HttpServiceImpl.class);

  @Resource
  private MetaResourceRepository             metaResourceRepository;

  private PoolingHttpClientConnectionManager cm;

  private CookieStore                        cookieStore;

  private CloseableHttpClient                httpClient;

  private RequestConfig                      requestConfig;

  @Resource
  private UniversalDetectorPlugin            universalDetectorPlugin;


  @Override
  public FetchHttpResponse submit(FetchHttpRequest request) throws CrawlerException {
    logger.debug("发送HTTP请求:{}",request.getUrl());
    long star = System.currentTimeMillis();
    //TODO 放入池中
    HttpClientContext context = HttpClientContext.create();
    FetchHttpResponse response = null;
    try{
      context.setRequestConfig(this.requestConfig);
      this.setProxy(context);
      if(HttpMethod.GET == request.getHttpMethod()){
        response = get(request,context);
      }else if(HttpMethod.POST== request.getHttpMethod()){
        response = post(request,context);
      }else{
        throw new CrawlerException("HTTP请求方式["+request.getHttpMethod().getCode()+"]暂不支持.");
      }
    }catch(Exception e){
      throw new CrawlerException(e);
    }finally{
      logger.info("发送HTTP请求结束,状态码:{},{},耗时:{}ms",response == null ? null : response.getStatusCode(),request.getUrl(),(System.currentTimeMillis() - star));
    }
    return response;
  }

  /**
   * 以GET方式发送Http请求
   * @param request
   * @param context
   * @return
   * @throws Exception
   */
  private FetchHttpResponse get(FetchHttpRequest request,HttpClientContext context) throws Exception{
    String url = FetchHelper.appendParam2URL(request.getUrl(),request.getParameter());
    HttpGet httpget = new HttpGet(url);
    CloseableHttpResponse response = null;
    try{
      this.setAgent(httpget);
      response = httpClient.execute(httpget, context);

      //是否需要重定向
      if(requestConfig.isRedirectsEnabled() && FetchHelper.needRedirectRequest(response)){
        String redirectAddr = this.getRedirectURL(context,response);
        if(StringUtils.isNotBlank(redirectAddr)){
          request.setUrl(redirectAddr);
          return this.get(request,context);
        }
      }
      return this.buildResponse(request,response,cookieStore);
    }catch (UnknownHostException e){
      logger.error("访问异常:{}.",request.getUrl(),e.getMessage());
      return this.buildResponse(request,response,cookieStore);
    }catch (CertificateException e){
      logger.error("访问异常:{}.",request.getUrl(),e.getMessage());
      return this.buildResponse(request,response,cookieStore);
    }catch(IOException e){
      logger.debug("访问超时:{}.",request.getUrl());
      return this.buildResponse(request,response,cookieStore);
    }catch(Exception e){
      throw e;
    }finally {
      httpget.releaseConnection();
      this.closeResources(response);
    }
  }

  /**
   * 以POST方式发送Http请求
   * @param request
   * @param context
   * @return
   * @throws Exception
   */
  private FetchHttpResponse post(FetchHttpRequest request,HttpClientContext context) throws Exception {
    HttpPost httppost = new HttpPost(request.getUrl());
    CloseableHttpResponse response = null;
    try {
      List<NameValuePair> postParams = new ArrayList<NameValuePair>();
      if (request.getParameter() != null && CollectionUtils.isNotEmpty(request.getParameter().getEntryList())) {
        for (Kvp kvp : request.getParameter().getEntryList()) {
          postParams.add(new BasicNameValuePair(kvp.getKey(), (String) kvp.getValue()));
        }
        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(postParams);
        httppost.setEntity(entity);
      }
      this.setAgent(httppost);
      response = httpClient.execute(httppost, context);
      //是否需要重定向
      if (requestConfig.isRedirectsEnabled() && FetchHelper.needRedirectRequest(response)) {
        String redirectAddr = this.getRedirectURL(context, response);
        if (StringUtils.isNotBlank(redirectAddr)) {
          request.setUrl(redirectAddr);
          return this.post(request, context);
        }
      }
      return this.buildResponse(request, response, cookieStore);
    }catch (UnknownHostException e){
      logger.error("访问异常:{}.",request.getUrl(),e.getMessage());
      return this.buildResponse(request,response,cookieStore);
    }catch (CertificateException e){
      logger.error("访问异常:{}.",request.getUrl(),e.getMessage());
      return this.buildResponse(request,response,cookieStore);
    }catch(IOException e){
      logger.debug("访问超时:{}.",request.getUrl());
      return this.buildResponse(request,response,cookieStore);
    }catch(Exception e){
      throw e;
    }finally {
      httppost.releaseConnection();
      this.closeResources(response);
    }
  }

  /**
   * 转换HTTP请求结果
   * </br>Created on 2016年4月29日
   * @param orgiRequest
   * @param response
   * @param cookieStore
   * @return
   * @throws Exception
   */
  private FetchHttpResponse buildResponse(FetchHttpRequest orgiRequest,CloseableHttpResponse response, CookieStore cookieStore) throws Exception{
    FetchHttpResponse resp = new FetchHttpResponse();

    //判断是否内部错误需要重试
    if(response == null){
      resp.setNeedRetry(true);
      return resp;
    }

    int statusCode = response.getStatusLine().getStatusCode();
    logger.debug("组装HTTP响应:响应码:{},{}.",statusCode,orgiRequest.getUrl());

    resp.setStatusCode(statusCode);
    resp.setNeedRetry(FetchHelper.needRetryRequest(response));
    resp.setNeedParse(FetchHelper.needParser(response));
    if(resp.isNeedRetry() || !resp.isNeedParse()){
      return resp;
    }

    HttpEntity entity = response.getEntity();
    byte[] content = EntityUtils.toByteArray(entity);
    resp.setCharset(this.getUniversalCode(entity,content));
    resp.setContent(content);
    resp.setCookies(cookieStore.getCookies());
    Map<String, String> headers = new HashMap<String, String>();
    for(Header h : response.getAllHeaders()){
      headers.put(h.getName(), h.getValue());
    }
    resp.setFile(FetchHelper.isFile(headers,metaResourceRepository.getHttpContentTypeWithFile()));
    resp.setHeaders(headers);
    EntityUtils.consume(entity);
    return resp;
  }

  /**
   * 设置HTTP请求Agent
   * @param httpUri
   * @throws Exception
   */
  private void setAgent(HttpRequestBase httpUri) throws Exception{
    List<String> agents = metaResourceRepository.getHttpBrowserAgents();
    String agent = CrawlerConstants.DEFAULT_HTTP_AGENT;
    if(CollectionUtils.isNotEmpty(agents)){
     agent = agents.get(RandomUtils.nextInt(agents.size()));
    }
    httpUri.setHeader(CrawlerConstants.USER_AGENT, agent);
  }

  /**
   * 设置HTTP请求代理
   * </br>Created on 2016年4月29日
   * @param httpContext
   */
  private void setProxy(HttpClientContext httpContext){
    boolean proxySwitch = metaResourceRepository.getHttpProxySwitch();
    if(!proxySwitch){
      return;
    }
    String proxyString = "";
    String proxyStringSplit = "";
    if(StringUtils.isBlank(proxyString) || StringUtils.isBlank(proxyStringSplit)){
      logger.error("HTTP代理配置或许为空,[{}]-[{}].",proxyString,proxyStringSplit);
      throw new CrawlerIllegalException("HTTP代理配置失败:"+proxyString);
    }
    String[] arr = StringUtils.split(proxyString,proxyStringSplit);
    if(arr.length != 4){
      throw new CrawlerIllegalException("HTTP代理格式配置不正确:"+proxyString);
    }
    HttpHost proxy = new HttpHost(arr[0], Integer.parseInt(arr[1]));
    BasicScheme proxyAuth = new BasicScheme();
    try {
      proxyAuth.processChallenge(new BasicHeader(AUTH.PROXY_AUTH, "BASIC realm=default"));
    } catch (MalformedChallengeException e) {
      logger.error("HTTP代理设置异常.",e);
      throw new IllegalArgumentException(e);
    }
    BasicAuthCache authCache = new BasicAuthCache();
    authCache.put(proxy, proxyAuth);
    CredentialsProvider credsProvider  = new BasicCredentialsProvider();
    credsProvider.setCredentials(new AuthScope(proxy),new UsernamePasswordCredentials(arr[2], arr[3]));
    httpContext.setAuthCache(authCache);
    httpContext.setCredentialsProvider(credsProvider);
  }

  /**
   * 获取跳转地址
   * @param context
   * @param response
   * @return
   */
  private String getRedirectURL(HttpClientContext context,CloseableHttpResponse response){
    List<URI> redirects = context.getRedirectLocations();
    String redirectAddr = null;
    if(CollectionUtils.isEmpty(redirects)){
      Header locationHeader = response.getFirstHeader("Location");
      redirectAddr = locationHeader.getValue();
    }else{
      try {
        redirectAddr = redirects.get(0).toURL().toString();
      } catch (Exception e) {
        logger.warn("获取跳转地址异常.",e);
      }
    }
    return redirectAddr;
  }

  private void closeResources(CloseableHttpResponse response) {
    logger.debug("释放Http响应资源");
    try{
      if(response != null){
        response.close();
      }
      if(cookieStore != null){
        cookieStore.clear();
      }
    }catch(IOException e){
      logger.warn("释放Http响应资源异常.",e);
    }
  }

  /**
   * 获取编码格式
   * 1.Http Header中的content-type
   * 2.页面Meta Charset
   * 3.页头中Document定义
   * @return
   */
  private Charset getUniversalCode(HttpEntity entity,byte[] content){
    Charset charset = FetchHelper.getCharset(entity);
    if(charset != null){
      return charset;
    }
    return universalDetectorPlugin.detector(content);
  }

  @PostConstruct
  public void initilizer() throws Exception {
    logger.info("初始化Http客户端");
    int httpConnectTimeout = metaResourceRepository.getHttpConnectTimeout();
    int httpConnectSocketTimeout = metaResourceRepository.getHttpConnectSocketTimeout();
    int httpConnectMaxTotal = metaResourceRepository.getHttpConnectMaxTotal();
    int httpRetryCount = metaResourceRepository.getHttpRetryTimes();
    boolean allowRedirect = metaResourceRepository.getHttpAllowRedirect();

    cookieStore = new BasicCookieStore();

    cm = new PoolingHttpClientConnectionManager();
    cm.setMaxTotal(httpConnectMaxTotal);
    cm.setDefaultMaxPerRoute(cm.getMaxTotal());

    httpClient = HttpClients.custom()
        .setConnectionManager(cm)
        .setRedirectStrategy(new LaxRedirectStrategy())
        .setDefaultCookieStore(cookieStore)
        .setRetryHandler(new HttpRequestRetryHandler() {
          @Override
          public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
            if(executionCount >= httpRetryCount){
              return false;
            }
            if(exception instanceof NoHttpResponseException){
              return true;
            }
            if(exception instanceof ConnectionPoolTimeoutException){
              return true;
            }
            if(exception instanceof SocketTimeoutException){
              return true;
            }
            if(exception instanceof SSLHandshakeException){
              return false;
            }
            HttpRequest request = (HttpRequest) context.getAttribute(
                HttpCoreContext.HTTP_REQUEST);
            boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
            if(idempotent){
              return true;
            }
            return false;
          }
        })
        .build();

    requestConfig = RequestConfig.custom()
        .setSocketTimeout(httpConnectSocketTimeout)
        .setConnectTimeout(httpConnectTimeout)
        .setConnectionRequestTimeout(httpConnectTimeout)
        .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY)
        .setRedirectsEnabled(allowRedirect)
        .setCircularRedirectsAllowed(false)
        .build();
  }
}
